{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Could not load the custom kernel for multi-scale deformable attention: /home/dengyunfei/.cache/torch_extensions/py311_cu124/MultiScaleDeformableAttention/MultiScaleDeformableAttention.so: cannot open shared object file: No such file or directory\n",
      "Could not load the custom kernel for multi-scale deformable attention: /home/dengyunfei/.cache/torch_extensions/py311_cu124/MultiScaleDeformableAttention/MultiScaleDeformableAttention.so: cannot open shared object file: No such file or directory\n",
      "Could not load the custom kernel for multi-scale deformable attention: /home/dengyunfei/.cache/torch_extensions/py311_cu124/MultiScaleDeformableAttention/MultiScaleDeformableAttention.so: cannot open shared object file: No such file or directory\n",
      "Could not load the custom kernel for multi-scale deformable attention: /home/dengyunfei/.cache/torch_extensions/py311_cu124/MultiScaleDeformableAttention/MultiScaleDeformableAttention.so: cannot open shared object file: No such file or directory\n",
      "Could not load the custom kernel for multi-scale deformable attention: /home/dengyunfei/.cache/torch_extensions/py311_cu124/MultiScaleDeformableAttention/MultiScaleDeformableAttention.so: cannot open shared object file: No such file or directory\n",
      "Could not load the custom kernel for multi-scale deformable attention: /home/dengyunfei/.cache/torch_extensions/py311_cu124/MultiScaleDeformableAttention/MultiScaleDeformableAttention.so: cannot open shared object file: No such file or directory\n",
      "Could not load the custom kernel for multi-scale deformable attention: /home/dengyunfei/.cache/torch_extensions/py311_cu124/MultiScaleDeformableAttention/MultiScaleDeformableAttention.so: cannot open shared object file: No such file or directory\n",
      "Could not load the custom kernel for multi-scale deformable attention: /home/dengyunfei/.cache/torch_extensions/py311_cu124/MultiScaleDeformableAttention/MultiScaleDeformableAttention.so: cannot open shared object file: No such file or directory\n",
      "Could not load the custom kernel for multi-scale deformable attention: /home/dengyunfei/.cache/torch_extensions/py311_cu124/MultiScaleDeformableAttention/MultiScaleDeformableAttention.so: cannot open shared object file: No such file or directory\n",
      "Could not load the custom kernel for multi-scale deformable attention: /home/dengyunfei/.cache/torch_extensions/py311_cu124/MultiScaleDeformableAttention/MultiScaleDeformableAttention.so: cannot open shared object file: No such file or directory\n",
      "Could not load the custom kernel for multi-scale deformable attention: /home/dengyunfei/.cache/torch_extensions/py311_cu124/MultiScaleDeformableAttention/MultiScaleDeformableAttention.so: cannot open shared object file: No such file or directory\n",
      "Could not load the custom kernel for multi-scale deformable attention: /home/dengyunfei/.cache/torch_extensions/py311_cu124/MultiScaleDeformableAttention/MultiScaleDeformableAttention.so: cannot open shared object file: No such file or directory\n",
      "Some weights of the model checkpoint at /media/dengyunfei/6T/data/models/huggingface/grounding-dino-tiny were not used when initializing GroundingDinoForObjectDetection: ['model.decoder.layers.0.encoder_attn_text.in_proj_bias', 'model.decoder.layers.0.encoder_attn_text.in_proj_weight', 'model.decoder.layers.0.self_attn.in_proj_bias', 'model.decoder.layers.0.self_attn.in_proj_weight', 'model.decoder.layers.1.encoder_attn_text.in_proj_bias', 'model.decoder.layers.1.encoder_attn_text.in_proj_weight', 'model.decoder.layers.1.self_attn.in_proj_bias', 'model.decoder.layers.1.self_attn.in_proj_weight', 'model.decoder.layers.2.encoder_attn_text.in_proj_bias', 'model.decoder.layers.2.encoder_attn_text.in_proj_weight', 'model.decoder.layers.2.self_attn.in_proj_bias', 'model.decoder.layers.2.self_attn.in_proj_weight', 'model.decoder.layers.3.encoder_attn_text.in_proj_bias', 'model.decoder.layers.3.encoder_attn_text.in_proj_weight', 'model.decoder.layers.3.self_attn.in_proj_bias', 'model.decoder.layers.3.self_attn.in_proj_weight', 'model.decoder.layers.4.encoder_attn_text.in_proj_bias', 'model.decoder.layers.4.encoder_attn_text.in_proj_weight', 'model.decoder.layers.4.self_attn.in_proj_bias', 'model.decoder.layers.4.self_attn.in_proj_weight', 'model.decoder.layers.5.encoder_attn_text.in_proj_bias', 'model.decoder.layers.5.encoder_attn_text.in_proj_weight', 'model.decoder.layers.5.self_attn.in_proj_bias', 'model.decoder.layers.5.self_attn.in_proj_weight', 'model.encoder.layers.0.text_enhancer_layer.self_attn.in_proj_bias', 'model.encoder.layers.0.text_enhancer_layer.self_attn.in_proj_weight', 'model.encoder.layers.1.text_enhancer_layer.self_attn.in_proj_bias', 'model.encoder.layers.1.text_enhancer_layer.self_attn.in_proj_weight', 'model.encoder.layers.2.text_enhancer_layer.self_attn.in_proj_bias', 'model.encoder.layers.2.text_enhancer_layer.self_attn.in_proj_weight', 'model.encoder.layers.3.text_enhancer_layer.self_attn.in_proj_bias', 'model.encoder.layers.3.text_enhancer_layer.self_attn.in_proj_weight', 'model.encoder.layers.4.text_enhancer_layer.self_attn.in_proj_bias', 'model.encoder.layers.4.text_enhancer_layer.self_attn.in_proj_weight', 'model.encoder.layers.5.text_enhancer_layer.self_attn.in_proj_bias', 'model.encoder.layers.5.text_enhancer_layer.self_attn.in_proj_weight', 'model.text_backbone.pooler.dense.bias', 'model.text_backbone.pooler.dense.weight']\n",
      "- This IS expected if you are initializing GroundingDinoForObjectDetection from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing GroundingDinoForObjectDetection from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of GroundingDinoForObjectDetection were not initialized from the model checkpoint at /media/dengyunfei/6T/data/models/huggingface/grounding-dino-tiny and are newly initialized: ['model.decoder.layers.0.encoder_attn_text.key.bias', 'model.decoder.layers.0.encoder_attn_text.key.weight', 'model.decoder.layers.0.encoder_attn_text.query.bias', 'model.decoder.layers.0.encoder_attn_text.query.weight', 'model.decoder.layers.0.encoder_attn_text.value.bias', 'model.decoder.layers.0.encoder_attn_text.value.weight', 'model.decoder.layers.0.self_attn.key.bias', 'model.decoder.layers.0.self_attn.key.weight', 'model.decoder.layers.0.self_attn.query.bias', 'model.decoder.layers.0.self_attn.query.weight', 'model.decoder.layers.0.self_attn.value.bias', 'model.decoder.layers.0.self_attn.value.weight', 'model.decoder.layers.1.encoder_attn_text.key.bias', 'model.decoder.layers.1.encoder_attn_text.key.weight', 'model.decoder.layers.1.encoder_attn_text.query.bias', 'model.decoder.layers.1.encoder_attn_text.query.weight', 'model.decoder.layers.1.encoder_attn_text.value.bias', 'model.decoder.layers.1.encoder_attn_text.value.weight', 'model.decoder.layers.1.self_attn.key.bias', 'model.decoder.layers.1.self_attn.key.weight', 'model.decoder.layers.1.self_attn.query.bias', 'model.decoder.layers.1.self_attn.query.weight', 'model.decoder.layers.1.self_attn.value.bias', 'model.decoder.layers.1.self_attn.value.weight', 'model.decoder.layers.2.encoder_attn_text.key.bias', 'model.decoder.layers.2.encoder_attn_text.key.weight', 'model.decoder.layers.2.encoder_attn_text.query.bias', 'model.decoder.layers.2.encoder_attn_text.query.weight', 'model.decoder.layers.2.encoder_attn_text.value.bias', 'model.decoder.layers.2.encoder_attn_text.value.weight', 'model.decoder.layers.2.self_attn.key.bias', 'model.decoder.layers.2.self_attn.key.weight', 'model.decoder.layers.2.self_attn.query.bias', 'model.decoder.layers.2.self_attn.query.weight', 'model.decoder.layers.2.self_attn.value.bias', 'model.decoder.layers.2.self_attn.value.weight', 'model.decoder.layers.3.encoder_attn_text.key.bias', 'model.decoder.layers.3.encoder_attn_text.key.weight', 'model.decoder.layers.3.encoder_attn_text.query.bias', 'model.decoder.layers.3.encoder_attn_text.query.weight', 'model.decoder.layers.3.encoder_attn_text.value.bias', 'model.decoder.layers.3.encoder_attn_text.value.weight', 'model.decoder.layers.3.self_attn.key.bias', 'model.decoder.layers.3.self_attn.key.weight', 'model.decoder.layers.3.self_attn.query.bias', 'model.decoder.layers.3.self_attn.query.weight', 'model.decoder.layers.3.self_attn.value.bias', 'model.decoder.layers.3.self_attn.value.weight', 'model.decoder.layers.4.encoder_attn_text.key.bias', 'model.decoder.layers.4.encoder_attn_text.key.weight', 'model.decoder.layers.4.encoder_attn_text.query.bias', 'model.decoder.layers.4.encoder_attn_text.query.weight', 'model.decoder.layers.4.encoder_attn_text.value.bias', 'model.decoder.layers.4.encoder_attn_text.value.weight', 'model.decoder.layers.4.self_attn.key.bias', 'model.decoder.layers.4.self_attn.key.weight', 'model.decoder.layers.4.self_attn.query.bias', 'model.decoder.layers.4.self_attn.query.weight', 'model.decoder.layers.4.self_attn.value.bias', 'model.decoder.layers.4.self_attn.value.weight', 'model.decoder.layers.5.encoder_attn_text.key.bias', 'model.decoder.layers.5.encoder_attn_text.key.weight', 'model.decoder.layers.5.encoder_attn_text.query.bias', 'model.decoder.layers.5.encoder_attn_text.query.weight', 'model.decoder.layers.5.encoder_attn_text.value.bias', 'model.decoder.layers.5.encoder_attn_text.value.weight', 'model.decoder.layers.5.self_attn.key.bias', 'model.decoder.layers.5.self_attn.key.weight', 'model.decoder.layers.5.self_attn.query.bias', 'model.decoder.layers.5.self_attn.query.weight', 'model.decoder.layers.5.self_attn.value.bias', 'model.decoder.layers.5.self_attn.value.weight', 'model.encoder.layers.0.text_enhancer_layer.self_attn.key.bias', 'model.encoder.layers.0.text_enhancer_layer.self_attn.key.weight', 'model.encoder.layers.0.text_enhancer_layer.self_attn.query.bias', 'model.encoder.layers.0.text_enhancer_layer.self_attn.query.weight', 'model.encoder.layers.0.text_enhancer_layer.self_attn.value.bias', 'model.encoder.layers.0.text_enhancer_layer.self_attn.value.weight', 'model.encoder.layers.1.text_enhancer_layer.self_attn.key.bias', 'model.encoder.layers.1.text_enhancer_layer.self_attn.key.weight', 'model.encoder.layers.1.text_enhancer_layer.self_attn.query.bias', 'model.encoder.layers.1.text_enhancer_layer.self_attn.query.weight', 'model.encoder.layers.1.text_enhancer_layer.self_attn.value.bias', 'model.encoder.layers.1.text_enhancer_layer.self_attn.value.weight', 'model.encoder.layers.2.text_enhancer_layer.self_attn.key.bias', 'model.encoder.layers.2.text_enhancer_layer.self_attn.key.weight', 'model.encoder.layers.2.text_enhancer_layer.self_attn.query.bias', 'model.encoder.layers.2.text_enhancer_layer.self_attn.query.weight', 'model.encoder.layers.2.text_enhancer_layer.self_attn.value.bias', 'model.encoder.layers.2.text_enhancer_layer.self_attn.value.weight', 'model.encoder.layers.3.text_enhancer_layer.self_attn.key.bias', 'model.encoder.layers.3.text_enhancer_layer.self_attn.key.weight', 'model.encoder.layers.3.text_enhancer_layer.self_attn.query.bias', 'model.encoder.layers.3.text_enhancer_layer.self_attn.query.weight', 'model.encoder.layers.3.text_enhancer_layer.self_attn.value.bias', 'model.encoder.layers.3.text_enhancer_layer.self_attn.value.weight', 'model.encoder.layers.4.text_enhancer_layer.self_attn.key.bias', 'model.encoder.layers.4.text_enhancer_layer.self_attn.key.weight', 'model.encoder.layers.4.text_enhancer_layer.self_attn.query.bias', 'model.encoder.layers.4.text_enhancer_layer.self_attn.query.weight', 'model.encoder.layers.4.text_enhancer_layer.self_attn.value.bias', 'model.encoder.layers.4.text_enhancer_layer.self_attn.value.weight', 'model.encoder.layers.5.text_enhancer_layer.self_attn.key.bias', 'model.encoder.layers.5.text_enhancer_layer.self_attn.key.weight', 'model.encoder.layers.5.text_enhancer_layer.self_attn.query.bias', 'model.encoder.layers.5.text_enhancer_layer.self_attn.query.weight', 'model.encoder.layers.5.text_enhancer_layer.self_attn.value.bias', 'model.encoder.layers.5.text_enhancer_layer.self_attn.value.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processer <class 'transformers.models.grounding_dino.processing_grounding_dino.GroundingDinoProcessor'>\n",
      "model <class 'transformers.models.grounding_dino.modeling_grounding_dino.GroundingDinoForObjectDetection'>\n",
      "输入处理 dict_keys(['input_ids', 'token_type_ids', 'attention_mask', 'pixel_values', 'pixel_mask'])\n",
      "tensor([[ 101, 1037, 6302, 1997, 8870, 1012,  102]], device='cuda:0')\n",
      "tensor([[0, 0, 0, 0, 0, 0, 0]], device='cuda:0')\n",
      "torch.Size([1, 7])\n",
      "torch.Size([1, 3, 800, 1066])\n",
      "torch.Size([1, 800, 1066])\n",
      "输出处理\n",
      "odict_keys(['logits', 'pred_boxes', 'last_hidden_state', 'init_reference_points', 'intermediate_hidden_states', 'intermediate_reference_points', 'encoder_last_hidden_state_vision', 'encoder_last_hidden_state_text', 'enc_outputs_class', 'enc_outputs_coord_logits'])\n",
      "torch.Size([1, 900, 256])\n",
      "torch.Size([1, 900, 4])\n",
      "torch.Size([1, 17821, 256])\n",
      "torch.Size([1, 7, 256])\n",
      "torch.Size([1, 17821, 256])\n",
      "torch.Size([1, 17821, 4])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'scores': tensor([], device='cuda:0'),\n",
       "  'labels': [],\n",
       "  'boxes': tensor([], device='cuda:0', size=(0, 4))}]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "import requests\n",
    "\n",
    "import torch\n",
    "from PIL import Image\n",
    "from transformers import AutoProcessor, AutoModelForZeroShotObjectDetection \n",
    "\n",
    "# model_id = \"IDEA-Research/grounding-dino-tiny\"\n",
    "model_id = \"/media/dengyunfei/6T/data/models/huggingface/grounding-dino-tiny\"\n",
    "device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n",
    "# print(torch.cuda.is_available())\n",
    "\n",
    "processor = AutoProcessor.from_pretrained(model_id)\n",
    "model = AutoModelForZeroShotObjectDetection.from_pretrained(model_id).to(device)\n",
    "\n",
    "print(\"processer {}\".format(processor.__class__))\n",
    "print(\"model {}\".format(model.__class__))\n",
    "# processer <class 'transformers.models.grounding_dino.processing_grounding_dino.GroundingDinoProcessor'>\n",
    "# model <class 'transformers.models.grounding_dino.modeling_grounding_dino.GroundingDinoForObjectDetection'>\n",
    "\n",
    "image_url = \"http://images.cocodataset.org/val2017/000000039769.jpg\"\n",
    "image = Image.open(requests.get(image_url, stream=True).raw)\n",
    "# Check for cats and remote controls\n",
    "# VERY important: text queries need to be lowercased + end with a dot\n",
    "# 注意：输入的文字必须是小写，且其中必须以英文句点结尾。\n",
    "text = \"a photo of cats.\" \n",
    "\n",
    "inputs = processor(images=image, text=text, return_tensors=\"pt\").to(device)\n",
    "\n",
    "print(\"输入处理\",inputs.keys())\n",
    "print(inputs['input_ids'])\n",
    "print(inputs['token_type_ids'])\n",
    "print(inputs['attention_mask'].shape)\n",
    "print(inputs['pixel_values'].shape)\n",
    "print(inputs['pixel_mask'].shape)\n",
    "'''\n",
    "输入处理 dict_keys(['input_ids', 'token_type_ids', 'attention_mask', 'pixel_values', 'pixel_mask'])\n",
    "tensor([[ 101, 1037, 6302, 1997, 8870, 1012,  102]], device='cuda:0')\n",
    "tensor([[0, 0, 0, 0, 0, 0, 0]], device='cuda:0')\n",
    "torch.Size([1, 7])\n",
    "torch.Size([1, 3, 800, 1066])\n",
    "torch.Size([1, 800, 1066])\n",
    "\n",
    "'''\n",
    "\n",
    "with torch.no_grad():\n",
    "    outputs = model(**inputs)\n",
    "print(\"输出处理\")\n",
    "print(outputs.keys())\n",
    "print(outputs['logits'].shape)\n",
    "print(outputs['pred_boxes'].shape)\n",
    "print(outputs['encoder_last_hidden_state_vision'].shape)\n",
    "print(outputs['encoder_last_hidden_state_text'].shape) # 将输入文字进行编码后的结果\n",
    "print(outputs['enc_outputs_class'].shape)\n",
    "print(outputs['enc_outputs_coord_logits'].shape)\n",
    "'''\n",
    "torch.Size([1, 900, 256])\n",
    "torch.Size([1, 900, 4])\n",
    "torch.Size([1, 17821, 256])\n",
    "torch.Size([1, 7, 256])   \n",
    "torch.Size([1, 17821, 256])\n",
    "torch.Size([1, 17821, 4])\n",
    "odict_keys([\n",
    "'logits', \n",
    "'pred_boxes', \n",
    "'last_hidden_state', \n",
    "'init_reference_points', \n",
    "'intermediate_hidden_states', \n",
    "'intermediate_reference_points', \n",
    "'encoder_last_hidden_state_vision', \n",
    "'encoder_last_hidden_state_text', \n",
    "'enc_outputs_class', \n",
    "'enc_outputs_coord_logits'\n",
    "])\n",
    "'''\n",
    "# \n",
    "results = processor.post_process_grounded_object_detection(\n",
    "    outputs,\n",
    "    inputs.input_ids,\n",
    "    box_threshold=0.4,\n",
    "    text_threshold=0.3,\n",
    "    target_sizes=[image.size[::-1]]\n",
    ")\n",
    "results\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch24",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
